{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Multi-LLM Integrations\n",
    "\n",
    "This notebook involves integrating multiple LLMs, a way to get comfortable working with LLM APIs.\n",
    "I'll be using Amazon Bedrock, which has a number of models that can be accessed via AWS SDK Boto3 library. I'll also use Deepseek directly via the API."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Importing required libraries\n",
    "# Boto3 library is AWS SDK for Python providing the necessary set of libraries (uv pip install boto3)\n",
    "\n",
    "import os\n",
    "import json\n",
    "import boto3\n",
    "from openai import OpenAI\n",
    "from dotenv import load_dotenv\n",
    "from IPython.display import Markdown, display"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Always remember to do this!\n",
    "load_dotenv(override=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Print the key prefixes to help with any debugging\n",
    "\n",
    "amazon_bedrock_bedrock_api_key = os.getenv('AMAZON_BEDROCK_API_KEY')\n",
    "deepseek_api_key = os.getenv('DEEPSEEK_API_KEY')\n",
    "\n",
    "if amazon_bedrock_bedrock_api_key:\n",
    "    print(f\"Amazon Bedrock API Key exists and begins {amazon_bedrock_bedrock_api_key[:4]}\")\n",
    "else:\n",
    "    print(\"Amazon Bedrock API Key not set (and this is optional)\")\n",
    "\n",
    "if deepseek_api_key:\n",
    "    print(f\"DeepSeek API Key exists and begins {deepseek_api_key[:3]}\")\n",
    "else:\n",
    "    print(\"DeepSeek API Key not set (and this is optional)\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Amazon Bedrock Client\n",
    "\n",
    "bedrock_client = boto3.client(\n",
    "    service_name=\"bedrock-runtime\",\n",
    "    region_name=\"us-east-1\"\n",
    ")\n",
    "\n",
    "# Deepseek Client\n",
    "\n",
    "deepseek_client = OpenAI(\n",
    "    api_key=deepseek_api_key, \n",
    "    base_url=\"https://api.deepseek.com\"\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Coming up with message for LLM Evaluation\n",
    "text = \"Please come up with a challenging, nuanced question that I can ask a number of LLMs to evaluate their intelligence. \"\n",
    "text += \"Answer only with the question, no explanation.\"\n",
    "messages = [{\"role\": \"user\", \"content\": [{\"text\": text}]}]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "messages"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Anthropic Claude 3.5 Sonnet for model evaluator question\n",
    "\n",
    "model_id = \"anthropic.claude-3-5-sonnet-20240620-v1:0\"\n",
    "response = bedrock_client.converse(\n",
    "    modelId=model_id,\n",
    "    messages=messages,\n",
    ")\n",
    "model_evaluator_question = response['output']['message']['content'][0]['text']\n",
    "print(model_evaluator_question)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "competitors = []\n",
    "answers = []\n",
    "messages = [{\"role\": \"user\", \"content\": model_evaluator_question}]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Deepseek chat model answer\n",
    "\n",
    "model_id = \"deepseek-chat\"\n",
    "response = deepseek_client.chat.completions.create(\n",
    "    model=model_id,\n",
    "    messages=messages\n",
    ")\n",
    "answer = response.choices[0].message.content\n",
    "\n",
    "display(Markdown(answer))\n",
    "competitors.append(model_id)\n",
    "answers.append(answer)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "messages = [{\"role\": \"user\", \"content\": [{\"text\": model_evaluator_question}]}]\n",
    "messages"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Amazon nova lite\n",
    "\n",
    "model_id = \"amazon.nova-lite-v1:0\"\n",
    "response = bedrock_client.converse(\n",
    "    modelId=model_id,\n",
    "    messages=messages,\n",
    ")\n",
    "answer = response['output']['message']['content'][0]['text']\n",
    "\n",
    "display(Markdown(answer))\n",
    "competitors.append(model_id)\n",
    "answers.append(answer)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Amazon Nova Pro\n",
    "\n",
    "model_id = \"amazon.nova-pro-v1:0\"\n",
    "response = bedrock_client.converse(\n",
    "    modelId=model_id,\n",
    "    messages=messages,\n",
    ")\n",
    "answer = response['output']['message']['content'][0]['text']\n",
    "\n",
    "display(Markdown(answer))\n",
    "competitors.append(model_id)\n",
    "answers.append(answer)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "messages = [{\"role\": \"user\", \"content\": [{\"text\": model_evaluator_question}]}]\n",
    "messages"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Cohere Command Light\n",
    "\n",
    "model_id = \"cohere.command-light-text-v14\"\n",
    "response = bedrock_client.converse(\n",
    "    modelId=model_id,\n",
    "    messages=messages,\n",
    ")\n",
    "answer = response['output']['message']['content'][0]['text']\n",
    "\n",
    "display(Markdown(answer))\n",
    "competitors.append(model_id)\n",
    "answers.append(answer)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## For the next cell, we will use Ollama\n",
    "\n",
    "Ollama runs a local web service that gives an OpenAI compatible endpoint,  \n",
    "and runs models locally using high performance C++ code.\n",
    "\n",
    "If you don't have Ollama, install it here by visiting https://ollama.com then pressing Download and following the instructions.\n",
    "\n",
    "After it's installed, you should be able to visit here: http://localhost:11434 and see the message \"Ollama is running\"\n",
    "\n",
    "You might need to restart Cursor (and maybe reboot). Then open a Terminal (control+\\`) and run `ollama serve`\n",
    "\n",
    "Useful Ollama commands (run these in the terminal, or with an exclamation mark in this notebook):\n",
    "\n",
    "`ollama pull <model_id>` downloads a model locally    \n",
    "`ollama ls` lists all the models you've downloaded  \n",
    "`ollama rm <model_id>` deletes the specified model from your downloads   \n",
    "`ollama run <model_id>` pulls the model if it doesn't exist locally, and run it."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<table style=\"margin: 0; text-align: left; width:100%\">\n",
    "    <tr>\n",
    "        <td style=\"width: 150px; height: 150px; vertical-align: middle;\">\n",
    "            <img src=\"../../../assets/stop.png\" width=\"150\" height=\"150\" style=\"display: block;\" />\n",
    "        </td>\n",
    "        <td>\n",
    "            <h2 style=\"color:#ff7800;\">Important</h2>\n",
    "            <span style=\"color:#ff7800;\">The model called <b>llama3.3</b> is FAR too large for home computers - it's not intended for personal computing and will consume all your resources! Stick with the nicely sized <b>llama3.2</b> or <b>llama3.2:1b</b> and if you want larger, try llama3.1 or smaller variants of Qwen, Gemma, Phi or DeepSeek. See the <A href=\"https://ollama.com/models\">the Ollama models page</a> for a full list of models and sizes.\n",
    "            </span>\n",
    "        </td>\n",
    "    </tr>\n",
    "</table>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "!ollama run llama3.2:1b"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "messages = [{\"role\": \"user\", \"content\": model_evaluator_question}]\n",
    "messages"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "ollama = OpenAI(base_url='http://localhost:11434/v1', api_key='ollama')\n",
    "model_id = \"llama3.2:1b\"\n",
    "\n",
    "response = ollama.chat.completions.create(\n",
    "    model=model_id, \n",
    "    messages=messages\n",
    ")\n",
    "answer = response.choices[0].message.content\n",
    "\n",
    "display(Markdown(answer))\n",
    "competitors.append(model_id)\n",
    "answers.append(answer)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Listing all models and their answers\n",
    "print(competitors)\n",
    "print(answers)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Mapping each model with it's solution for the model evaluator question\n",
    "for competitor, answer in zip(competitors, answers):\n",
    "    print(f\"Competitor: {competitor}\\n\\n{answer}\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Masking out the model name for evaluation purposes - note the use of \"enumerate\"\n",
    "\n",
    "together = \"\"\n",
    "for index, answer in enumerate(answers):\n",
    "    together += f\"# Response from competitor {index+1}\\n\\n\"\n",
    "    together += answer + \"\\n\\n\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "print(together)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "judge = f\"\"\"You are judging a competition between {len(competitors)} competitors.\n",
    "Each model has been given this question:\n",
    "\n",
    "{model_evaluator_question}\n",
    "\n",
    "Your job is to evaluate each response for clarity and strength of argument, and rank them in order of best to worst.\n",
    "Respond with JSON, and only JSON, with the following format:\n",
    "{{\"results\": [\"best competitor number\", \"second best competitor number\", \"third best competitor number\", ...]}}\n",
    "\n",
    "Here are the responses from each competitor:\n",
    "\n",
    "{together}\n",
    "\n",
    "Now respond with the JSON with the ranked order of the competitors, nothing else. Do not include markdown formatting or code blocks.\"\"\"\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "print(judge)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "judge_messages = [{\"role\": \"user\", \"content\": [{\"text\": judge}]}]\n",
    "judge_messages"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Anthropic Claude 3.5 Sonnet for model evaluator question\n",
    "\n",
    "model_id = \"anthropic.claude-3-5-sonnet-20240620-v1:0\"\n",
    "response = bedrock_client.converse(\n",
    "    modelId=model_id,\n",
    "    messages=judge_messages,\n",
    ")\n",
    "model_evaluator_response = response['output']['message']['content'][0]['text']\n",
    "print(model_evaluator_response)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# OK let's turn this into results!\n",
    "\n",
    "results_dict = json.loads(model_evaluator_response)\n",
    "ranks = results_dict[\"results\"]\n",
    "for index, result in enumerate(ranks):\n",
    "    competitor = competitors[int(result)-1]\n",
    "    print(f\"Rank {index+1}: {competitor}\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<table style=\"margin: 0; text-align: left; width:100%\">\n",
    "    <tr>\n",
    "        <td style=\"width: 150px; height: 150px; vertical-align: middle;\">\n",
    "            <img src=\"../../../assets/business.png\" width=\"150\" height=\"150\" style=\"display: block;\" />\n",
    "        </td>\n",
    "        <td>\n",
    "            <h2 style=\"color:#00bfff;\">Commercial implications</h2>\n",
    "            <span style=\"color:#00bfff;\">These kinds of patterns - to send a task to multiple models, and evaluate results,\n",
    "            are common where you need to improve the quality of your LLM response. This approach can be universally applied\n",
    "            to business projects where accuracy is critical.\n",
    "            </span>\n",
    "        </td>\n",
    "    </tr>\n",
    "</table>"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": ".venv",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.12.1"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
